# -*- coding:utf-8 -*-
from pymongo import MongoClient
from config import config
from model import queue
from urllib2 import urlopen
from urllib2 import URLError, HTTPError
import urllib2
from bs4 import BeautifulSoup
from urlparse import urljoin
from urlparse import urldefrag
from time import sleep
import string
import random


def filter_url(url):
    if not url.startswith(config.base_url):
        return False
    return True


def url_format(url):
    url = urldefrag(url)[0]
    if "?icn=" in url:
        return url[0:url.find("?icn=")]
    # print url
    # url = url[len(config.base_url):len(url)]
    # print url
    return url


def get_html(url):
    try:
        html = urlopen(url)
        html = html.read()
    except HTTPError as e:
        return e
    except URLError as e:
        return e
    return html


def get_urls(url, html):
    result = set()
    bs_obj = BeautifulSoup(html, 'lxml')
    for link in bs_obj.findAll("a"):
        if 'href' in link.attrs:
            href = link.attrs['href']
            href = urljoin(url, href)
            if href not in result and filter_url(href):
                # print href
                href = url_format(href)
                # print href
                result.add(href)
    return result


def insert_to_queue(mongodb_table_queue, url_set):
    for url in url_set:
        url = url[len(config.base_url):len(url)]
        queue_item = mongodb_table_queue.find_one({"url":url})
        if queue_item is None:
            data = {
                'url': url,
                'status': queue.status_add,
                'retry_count': 0
            }
            print mongodb_table_queue.insert_one(data).inserted_id


def do_it_one(mongodb_db):
    mongodb_table_queue = mongodb_db[config.mongodb_table_queue]
    queue_item = mongodb_table_queue.find_one(
        {
            'status':queue.status_add,
            'url':{
                '$regex':'^/subject/\d+/reviews'
            }
        }
    )
    if queue_item is None:
        print "queue is empty"
        return
    url = config.base_url+queue_item['url']
    print url

    content = get_html(url)
    if type(content) is URLError or type(content) is HTTPError:
        print 'error'
        sleep(300)
        queue_item['status'] = queue.status_error
        queue_item['retry_count'] += 1
    else:
        print 'ok'
        queue_item['status'] = queue.status_ok
        mongodb_table_source_page = mongodb_db[config.mongodb_table_source_page]
        data = {
            'url': queue_item['url'],
            'content': content
        }
        print mongodb_table_source_page.insert_one(data).inserted_id
        url_set = get_urls(url, content)
        insert_to_queue(mongodb_table_queue, url_set)

    # 更新队列状态
    queue_item_id = queue_item['_id']
    del queue_item['_id']
    mongodb_table_queue.update_one({"_id":queue_item_id}, {"$set":queue_item})


def do_it(mongodb_db):
    while(1):
        do_it_one(mongodb_db)
        sleep(5)


def main():
    mongodb_client = MongoClient(config.mongodb_host, config.mongodb_port)
    mongodb_db = mongodb_client[config.mongodb_db]
    do_it(mongodb_db)


if __name__ == '__main__':
    main()
